{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import glob\n",
    "import json\n",
    "import os\n",
    "from pprint import pprint\n",
    "from sys_config import EXP_DIR\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "\n",
    "sns.set_style(\"white\")\n",
    "sns.set_context(\"notebook\")\n",
    "plt.rcParams['font.family'] = \"serif\"\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "\n",
    "\n",
    "# %matplotlib inline\n",
    "# %matplotlib notebook\n",
    "# %matplotlib widget\n",
    "%matplotlib ipympl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = glob.glob(EXP_DIR + f\"/experiments/efficiency/final.trans*/**\")\n",
    "experiments = glob.glob(EXP_DIR + f\"/experiments/main/final.trans*/**\") + experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Errno 2] No such file or directory: '/home/christos/PycharmProjects/seq3mt/experiments/experiments/main/final.trans.deen_postnorm_ls/20-04-13_09:13:03_21895060/BLEU.json'\n",
      "[Errno 2] No such file or directory: '/home/christos/PycharmProjects/seq3mt/experiments/experiments/main/final.trans.entr_postnorm_ls/20-04-13_09:13:03_21895091/BLEU.json'\n",
      "[Errno 2] No such file or directory: '/home/christos/PycharmProjects/seq3mt/experiments/experiments/main/final.trans.tren_postnorm_ls/20-04-13_09:13:03_21895080/BLEU.json'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bleu.test.1</th>\n",
       "      <th>bleu.test.5</th>\n",
       "      <th>bleu.val.1</th>\n",
       "      <th>bleu.val.5</th>\n",
       "      <th>config.batch_tokens</th>\n",
       "      <th>config.config</th>\n",
       "      <th>config.cores</th>\n",
       "      <th>config.data.backtranslate_path</th>\n",
       "      <th>config.data.lowercase</th>\n",
       "      <th>config.data.prior_path</th>\n",
       "      <th>...</th>\n",
       "      <th>metrics.epoch_ppl.VALID - MT</th>\n",
       "      <th>metrics.loss</th>\n",
       "      <th>metrics.loss.MT</th>\n",
       "      <th>metrics.loss.PRIOR</th>\n",
       "      <th>metrics.lr</th>\n",
       "      <th>metrics.ppl</th>\n",
       "      <th>metrics.ppl.MT</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>pair</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.2</td>\n",
       "      <td>5.6</td>\n",
       "      <td>4.7</td>\n",
       "      <td>4.9</td>\n",
       "      <td>10000</td>\n",
       "      <td>../../configs/transformer/trans.deen_base....</td>\n",
       "      <td>4</td>\n",
       "      <td>/rds/project/t2_vol4/rds-t2-cs055/cbaziotis/wo...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[8.979499459266663, 8.647571130232377, 8.44131...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[6.176323555016366e-08, 3.149925013058347e-06,...</td>\n",
       "      <td>[7938.657499410571, 5696.294360598452, 4634.65...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>final.trans.deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.3</td>\n",
       "      <td>5.8</td>\n",
       "      <td>4.7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>../../configs/transformer/trans.deen_base....</td>\n",
       "      <td>4</td>\n",
       "      <td>/rds/project/t2_vol4/rds-t2-cs055/cbaziotis/wo...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[8.979499459266663, 8.647571130232377, 8.44131...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[6.176323555016366e-08, 3.149925013058347e-06,...</td>\n",
       "      <td>[7938.657499410571, 5696.294360598452, 4634.65...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>deen_base_10000_fusion_3M</td>\n",
       "      <td>final.trans.deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.3</td>\n",
       "      <td>4.4</td>\n",
       "      <td>4.7</td>\n",
       "      <td>10000</td>\n",
       "      <td>../../configs/transformer/trans.deen_base....</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[8.962945063908895, 8.629665634848855, 8.42027...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[6.176323555016366e-08, 3.149925013058347e-06,...</td>\n",
       "      <td>[7808.319633393253, 5595.207097738077, 4538.15...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>final.trans.deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.2</td>\n",
       "      <td>5.4</td>\n",
       "      <td>4.4</td>\n",
       "      <td>4.8</td>\n",
       "      <td>10000</td>\n",
       "      <td>../../configs/transformer/trans.deen_base....</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[8.962945063908895, 8.629665634848855, 8.42027...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[6.176323555016366e-08, 3.149925013058347e-06,...</td>\n",
       "      <td>[7808.319633393253, 5595.207097738077, 4538.15...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>deen_base_10000_fusion_3M</td>\n",
       "      <td>final.trans.deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.2</td>\n",
       "      <td>5.4</td>\n",
       "      <td>4.6</td>\n",
       "      <td>4.8</td>\n",
       "      <td>10000</td>\n",
       "      <td>../../configs/transformer/trans.deen_base....</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[8.999773820241293, 8.658453161066229, 8.45190...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[6.176323555016366e-08, 3.149925013058347e-06,...</td>\n",
       "      <td>[8101.2513812581155, 5758.620111799558, 4684.0...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>final.trans.deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 109 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bleu.test.1  bleu.test.5  bleu.val.1  bleu.val.5  config.batch_tokens  \\\n",
       "0          5.2          5.6         4.7         4.9                10000   \n",
       "1          5.3          5.8         4.7         5.0                10000   \n",
       "2          5.0          5.3         4.4         4.7                10000   \n",
       "3          5.2          5.4         4.4         4.8                10000   \n",
       "4          5.2          5.4         4.6         4.8                10000   \n",
       "\n",
       "                                       config.config  config.cores  \\\n",
       "0  ../../configs/transformer/trans.deen_base....             4   \n",
       "1  ../../configs/transformer/trans.deen_base....             4   \n",
       "2  ../../configs/transformer/trans.deen_base....             4   \n",
       "3  ../../configs/transformer/trans.deen_base....             4   \n",
       "4  ../../configs/transformer/trans.deen_base....             4   \n",
       "\n",
       "                      config.data.backtranslate_path  config.data.lowercase  \\\n",
       "0  /rds/project/t2_vol4/rds-t2-cs055/cbaziotis/wo...                   True   \n",
       "1  /rds/project/t2_vol4/rds-t2-cs055/cbaziotis/wo...                   True   \n",
       "2                                                NaN                   True   \n",
       "3                                                NaN                   True   \n",
       "4                                                NaN                   True   \n",
       "\n",
       "  config.data.prior_path  ...  metrics.epoch_ppl.VALID - MT  \\\n",
       "0                   None  ...                           NaN   \n",
       "1                   None  ...                           NaN   \n",
       "2                   None  ...                           NaN   \n",
       "3                   None  ...                           NaN   \n",
       "4                   None  ...                           NaN   \n",
       "\n",
       "                                        metrics.loss metrics.loss.MT  \\\n",
       "0  [8.979499459266663, 8.647571130232377, 8.44131...             NaN   \n",
       "1  [8.979499459266663, 8.647571130232377, 8.44131...             NaN   \n",
       "2  [8.962945063908895, 8.629665634848855, 8.42027...             NaN   \n",
       "3  [8.962945063908895, 8.629665634848855, 8.42027...             NaN   \n",
       "4  [8.999773820241293, 8.658453161066229, 8.45190...             NaN   \n",
       "\n",
       "  metrics.loss.PRIOR                                         metrics.lr  \\\n",
       "0                NaN  [6.176323555016366e-08, 3.149925013058347e-06,...   \n",
       "1                NaN  [6.176323555016366e-08, 3.149925013058347e-06,...   \n",
       "2                NaN  [6.176323555016366e-08, 3.149925013058347e-06,...   \n",
       "3                NaN  [6.176323555016366e-08, 3.149925013058347e-06,...   \n",
       "4                NaN  [6.176323555016366e-08, 3.149925013058347e-06,...   \n",
       "\n",
       "                                         metrics.ppl metrics.ppl.MT  \\\n",
       "0  [7938.657499410571, 5696.294360598452, 4634.65...            NaN   \n",
       "1  [7938.657499410571, 5696.294360598452, 4634.65...            NaN   \n",
       "2  [7808.319633393253, 5595.207097738077, 4538.15...            NaN   \n",
       "3  [7808.319633393253, 5595.207097738077, 4538.15...            NaN   \n",
       "4  [8101.2513812581155, 5758.620111799558, 4684.0...            NaN   \n",
       "\n",
       "                       model                         name  pair  \n",
       "0            deen_base_10000  final.trans.deen_base_10000  deen  \n",
       "1  deen_base_10000_fusion_3M  final.trans.deen_base_10000  deen  \n",
       "2            deen_base_10000  final.trans.deen_base_10000  deen  \n",
       "3  deen_base_10000_fusion_3M  final.trans.deen_base_10000  deen  \n",
       "4            deen_base_10000  final.trans.deen_base_10000  deen  \n",
       "\n",
       "[5 rows x 109 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def flatten(d, parent_key='', sep='.'):\n",
    "    items = []\n",
    "    for k, v in d.items():\n",
    "        new_key = parent_key + sep + k if parent_key else k\n",
    "        if isinstance(v, collections.MutableMapping):\n",
    "            items.extend(flatten(v, new_key, sep=sep).items())\n",
    "        else:\n",
    "            items.append((new_key, v))\n",
    "    return dict(items)\n",
    "\n",
    "\n",
    "def read_bleu(exp, file):\n",
    "    with open(os.path.join(exp, file)) as json_file:\n",
    "        data = json_file.read()\n",
    "        data = data[:data.rfind(\"}\")+1]\n",
    "        bleu = flatten(json.loads(data), \"bleu\")\n",
    "        return bleu\n",
    "\n",
    "results = []\n",
    "for exp in sorted(experiments):\n",
    "    try:\n",
    "        name = exp.split(os.sep)[-2]\n",
    "        log = flatten(json.load(open(os.path.join(exp, f\"{name}.json\"))))\n",
    "        bleu = read_bleu(exp, \"BLEU.json\")\n",
    "        record = {**log, **bleu}\n",
    "        record[\"pair\"] = name.split(\"_\")[0].split(\".\")[-1]\n",
    "        record[\"model\"] = name.split(\".\")[-1]\n",
    "            \n",
    "        results.append(record)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    \n",
    "#     for size in [\"3M\", \"30M\"]:\n",
    "    for size in [\"3M\"]:\n",
    "        try:\n",
    "            name = exp.split(os.sep)[-2]\n",
    "            pair = name.split(\"_\")[0].split(\".\")[-1]\n",
    "            model = name.split(\".\")[-1] + f\"_fusion_{size}\"\n",
    "            log = flatten(json.load(open(os.path.join(exp, f\"{name}.json\"))))\n",
    "            bleu = read_bleu(exp, f\"BLEU_shallow_0.1_lm_news_{pair[2:]}_{size}_trans_big_best.json\")\n",
    "\n",
    "            record = {**log, **bleu}\n",
    "            record[\"pair\"] = pair\n",
    "            record[\"model\"] = model\n",
    "            results.append(record)\n",
    "        except Exception as e:\n",
    "#             print(e)\n",
    "            pass\n",
    "                \n",
    "df = pd.DataFrame.from_records(results)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>dev</th>\n",
       "      <th>model</th>\n",
       "      <th>pair</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.6</td>\n",
       "      <td>4.9</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.8</td>\n",
       "      <td>5.0</td>\n",
       "      <td>deen_base_10000_fusion_3M</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.3</td>\n",
       "      <td>4.7</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.4</td>\n",
       "      <td>4.8</td>\n",
       "      <td>deen_base_10000_fusion_3M</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.4</td>\n",
       "      <td>4.8</td>\n",
       "      <td>deen_base_10000</td>\n",
       "      <td>deen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test  dev                      model  pair\n",
       "0   5.6  4.9            deen_base_10000  deen\n",
       "1   5.8  5.0  deen_base_10000_fusion_3M  deen\n",
       "2   5.3  4.7            deen_base_10000  deen\n",
       "3   5.4  4.8  deen_base_10000_fusion_3M  deen\n",
       "4   5.4  4.8            deen_base_10000  deen"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the rest\n",
    "# keep_cols = [\"bleu.test.1\", \"bleu.test.5\", \"bleu.test.10\", \"bleu.val.1\", \"bleu.val.5\", \"bleu.val.10\", \"name\", \"pair\"]\n",
    "# keep_cols = [\"bleu.test.5\", \"bleu.test.10\", \"bleu.val.5\", \"bleu.val.10\", \"model\", \"pair\"]\n",
    "keep_cols = [\"bleu.test.5\", \"bleu.val.5\", \"model\", \"pair\"]\n",
    "keep = [k for k in df.columns if k in keep_cols]\n",
    "df = df[keep]\n",
    "df.rename(columns={'bleu.test.5': 'test', 'bleu.val.5': 'dev'}, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.columns[::-1]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "_std = lambda x: np.std(x, ddof=0)\n",
    "def eval_pair(pair):\n",
    "    _data = df[~df.model.str.contains(\"00\")]\n",
    "#     _data = _data[_data.model.str.contains(\"fusion\")]\n",
    "#     _data = df[df.model.str.contains(\"3M_kl_ls\")]\n",
    "    _data = _data[_data[\"pair\"]==pair].groupby([\"model\"], as_index=False).agg([np.mean, _std]).sort_values(by=\"model\", ascending=True).round(1)\n",
    "    display(HTML(_data.style.bar(color='#d65f5f').render()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 46.8%, transparent 46.8%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 40.0%, transparent 40.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 63.8%, transparent 63.8%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 55.0%, transparent 55.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 72.3%, transparent 72.3%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 75.0%, transparent 75.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 65.0%, transparent 65.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 87.5%, transparent 87.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 89.4%, transparent 89.4%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 80.0%, transparent 80.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.5%, transparent 37.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 34.0%, transparent 34.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.5%, transparent 37.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 31.7%, transparent 31.7%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 36.2%, transparent 36.2%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.3%, transparent 28.3%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 95.7%, transparent 95.7%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 91.7%, transparent 91.7%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.5%, transparent 37.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.5%, transparent 37.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 95.7%, transparent 95.7%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 95.0%, transparent 95.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 12.5%, transparent 12.5%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 96.7%, transparent 96.7%);\n",
       "        }    #T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }</style><table id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" colspan=2>dev</th>        <th class=\"col_heading level0 col2\" colspan=2>test</th>    </tr>    <tr>        <th class=\"blank level1\" ></th>        <th class=\"col_heading level1 col0\" >mean</th>        <th class=\"col_heading level1 col1\" ><lambda></th>        <th class=\"col_heading level1 col2\" >mean</th>        <th class=\"col_heading level1 col3\" ><lambda></th>    </tr>    <tr>        <th class=\"index_name level0\" >model</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row0\" class=\"row_heading level0 row0\" >deen_base</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col0\" class=\"data row0 col0\" >22.6</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col1\" class=\"data row0 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col2\" class=\"data row0 col2\" >26.9</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow0_col3\" class=\"data row0 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row1\" class=\"row_heading level0 row1\" >deen_base_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col0\" class=\"data row1 col0\" >23.4</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col1\" class=\"data row1 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col2\" class=\"data row1 col2\" >27.8</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow1_col3\" class=\"data row1 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row2\" class=\"row_heading level0 row2\" >deen_base_ls</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col0\" class=\"data row2 col0\" >23.8</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col1\" class=\"data row2 col1\" >0.6</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col2\" class=\"data row2 col2\" >28.4</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow2_col3\" class=\"data row2 col3\" >0.7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row3\" class=\"row_heading level0 row3\" >deen_base_ls_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col0\" class=\"data row3 col0\" >24.6</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col1\" class=\"data row3 col1\" >0.8</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col2\" class=\"data row3 col2\" >29.3</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow3_col3\" class=\"data row3 col3\" >0.8</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row4\" class=\"row_heading level0 row4\" >deen_postnorm</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col0\" class=\"data row4 col0\" >20.4</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col1\" class=\"data row4 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col2\" class=\"data row4 col2\" >24.5</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow4_col3\" class=\"data row4 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row5\" class=\"row_heading level0 row5\" >deen_postnorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col0\" class=\"data row5 col0\" >22</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col1\" class=\"data row5 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col2\" class=\"data row5 col2\" >26.4</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow5_col3\" class=\"data row5 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row6\" class=\"row_heading level0 row6\" >deen_prenorm</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col0\" class=\"data row6 col0\" >20.4</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col1\" class=\"data row6 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col2\" class=\"data row6 col2\" >24.5</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow6_col3\" class=\"data row6 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row7\" class=\"row_heading level0 row7\" >deen_prenorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col0\" class=\"data row7 col0\" >22.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col1\" class=\"data row7 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col2\" class=\"data row7 col2\" >26.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow7_col3\" class=\"data row7 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row8\" class=\"row_heading level0 row8\" >deen_prior_30M_kl</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col0\" class=\"data row8 col0\" >24.9</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col1\" class=\"data row8 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col2\" class=\"data row8 col2\" >30</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow8_col3\" class=\"data row8 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row9\" class=\"row_heading level0 row9\" >deen_prior_30M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col0\" class=\"data row9 col0\" >25.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col1\" class=\"data row9 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col2\" class=\"data row9 col2\" >30.5</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow9_col3\" class=\"data row9 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row10\" class=\"row_heading level0 row10\" >deen_prior_3M_kl</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col0\" class=\"data row10 col0\" >24.9</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col1\" class=\"data row10 col1\" >0</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col2\" class=\"data row10 col2\" >30.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow10_col3\" class=\"data row10 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bblevel0_row11\" class=\"row_heading level0 row11\" >deen_prior_3M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col0\" class=\"data row11 col0\" >25.1</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col1\" class=\"data row11 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col2\" class=\"data row11 col2\" >30.3</td>\n",
       "                        <td id=\"T_1cd8ce60_8a01_11ea_94fd_d0509993d9bbrow11_col3\" class=\"data row11 col3\" >0</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_pair(\"deen\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.8%, transparent 37.8%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 39.7%, transparent 39.7%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 14.3%, transparent 14.3%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 42.2%, transparent 42.2%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 44.9%, transparent 44.9%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 57.8%, transparent 57.8%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 61.5%, transparent 61.5%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.6%, transparent 28.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 64.4%, transparent 64.4%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 65.4%, transparent 65.4%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.6%, transparent 28.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 5.1%, transparent 5.1%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.6%, transparent 28.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 6.7%, transparent 6.7%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 10.3%, transparent 10.3%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 57.1%, transparent 57.1%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 14.3%, transparent 14.3%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 6.7%, transparent 6.7%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 11.5%, transparent 11.5%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 97.8%, transparent 97.8%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 75.0%, transparent 75.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 93.6%, transparent 93.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.6%, transparent 28.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 75.0%, transparent 75.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 86.7%, transparent 86.7%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 84.6%, transparent 84.6%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 85.7%, transparent 85.7%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 93.3%, transparent 93.3%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 75.0%, transparent 75.0%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 92.3%, transparent 92.3%);\n",
       "        }    #T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 85.7%, transparent 85.7%);\n",
       "        }</style><table id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" colspan=2>dev</th>        <th class=\"col_heading level0 col2\" colspan=2>test</th>    </tr>    <tr>        <th class=\"blank level1\" ></th>        <th class=\"col_heading level1 col0\" >mean</th>        <th class=\"col_heading level1 col1\" ><lambda></th>        <th class=\"col_heading level1 col2\" >mean</th>        <th class=\"col_heading level1 col3\" ><lambda></th>    </tr>    <tr>        <th class=\"index_name level0\" >model</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row0\" class=\"row_heading level0 row0\" >ende_base</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col0\" class=\"data row0 col0\" >18.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col1\" class=\"data row0 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col2\" class=\"data row0 col2\" >25.6</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow0_col3\" class=\"data row0 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row1\" class=\"row_heading level0 row1\" >ende_base_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col0\" class=\"data row1 col0\" >18.5</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col1\" class=\"data row1 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col2\" class=\"data row1 col2\" >26</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow1_col3\" class=\"data row1 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row2\" class=\"row_heading level0 row2\" >ende_base_ls</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col0\" class=\"data row2 col0\" >19.2</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col1\" class=\"data row2 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col2\" class=\"data row2 col2\" >27.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow2_col3\" class=\"data row2 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row3\" class=\"row_heading level0 row3\" >ende_base_ls_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col0\" class=\"data row3 col0\" >19.5</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col1\" class=\"data row3 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col2\" class=\"data row3 col2\" >27.6</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow3_col3\" class=\"data row3 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row4\" class=\"row_heading level0 row4\" >ende_postnorm</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col0\" class=\"data row4 col0\" >16.6</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col1\" class=\"data row4 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col2\" class=\"data row4 col2\" >22.9</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow4_col3\" class=\"data row4 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row5\" class=\"row_heading level0 row5\" >ende_postnorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col0\" class=\"data row5 col0\" >16.9</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col1\" class=\"data row5 col1\" >0.5</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col2\" class=\"data row5 col2\" >23.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow5_col3\" class=\"data row5 col3\" >0.5</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row6\" class=\"row_heading level0 row6\" >ende_prenorm</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col0\" class=\"data row6 col0\" >16.6</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col1\" class=\"data row6 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col2\" class=\"data row6 col2\" >22.5</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow6_col3\" class=\"data row6 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row7\" class=\"row_heading level0 row7\" >ende_prenorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col0\" class=\"data row7 col0\" >16.9</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col1\" class=\"data row7 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col2\" class=\"data row7 col2\" >23.4</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow7_col3\" class=\"data row7 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row8\" class=\"row_heading level0 row8\" >ende_prior_30M_kl</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col0\" class=\"data row8 col0\" >21</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col1\" class=\"data row8 col1\" >0.4</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col2\" class=\"data row8 col2\" >29.8</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow8_col3\" class=\"data row8 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row9\" class=\"row_heading level0 row9\" >ende_prior_30M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col0\" class=\"data row9 col0\" >21.1</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col1\" class=\"data row9 col1\" >0.4</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col2\" class=\"data row9 col2\" >30.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow9_col3\" class=\"data row9 col3\" >0.8</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row10\" class=\"row_heading level0 row10\" >ende_prior_3M_kl</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col0\" class=\"data row10 col0\" >20.5</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col1\" class=\"data row10 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col2\" class=\"data row10 col2\" >29.1</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow10_col3\" class=\"data row10 col3\" >0.7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bblevel0_row11\" class=\"row_heading level0 row11\" >ende_prior_3M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col0\" class=\"data row11 col0\" >20.8</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col1\" class=\"data row11 col1\" >0.4</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col2\" class=\"data row11 col2\" >29.7</td>\n",
       "                        <td id=\"T_1cd8ce61_8a01_11ea_94fd_d0509993d9bbrow11_col3\" class=\"data row11 col3\" >0.7</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_pair(\"ende\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 46.0%, transparent 46.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 39.6%, transparent 39.6%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 58.0%, transparent 58.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 54.2%, transparent 54.2%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 78.0%, transparent 78.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 77.1%, transparent 77.1%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 88.0%, transparent 88.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 87.5%, transparent 87.5%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 4.0%, transparent 4.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 2.1%, transparent 2.1%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 28.0%, transparent 28.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 27.1%, transparent 27.1%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 38.0%, transparent 38.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 27.1%, transparent 27.1%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 98.0%, transparent 98.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 95.8%, transparent 95.8%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 98.0%, transparent 98.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }</style><table id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" colspan=2>dev</th>        <th class=\"col_heading level0 col2\" colspan=2>test</th>    </tr>    <tr>        <th class=\"blank level1\" ></th>        <th class=\"col_heading level1 col0\" >mean</th>        <th class=\"col_heading level1 col1\" ><lambda></th>        <th class=\"col_heading level1 col2\" >mean</th>        <th class=\"col_heading level1 col3\" ><lambda></th>    </tr>    <tr>        <th class=\"index_name level0\" >model</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row0\" class=\"row_heading level0 row0\" >tren_base</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col0\" class=\"data row0 col0\" >15.9</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col1\" class=\"data row0 col1\" >0</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col2\" class=\"data row0 col2\" >16.6</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow0_col3\" class=\"data row0 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row1\" class=\"row_heading level0 row1\" >tren_base_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col0\" class=\"data row1 col0\" >16.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col1\" class=\"data row1 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col2\" class=\"data row1 col2\" >17.3</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow1_col3\" class=\"data row1 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row2\" class=\"row_heading level0 row2\" >tren_base_ls</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col0\" class=\"data row2 col0\" >17.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col1\" class=\"data row2 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col2\" class=\"data row2 col2\" >18.4</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow2_col3\" class=\"data row2 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row3\" class=\"row_heading level0 row3\" >tren_base_ls_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col0\" class=\"data row3 col0\" >18</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col1\" class=\"data row3 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col2\" class=\"data row3 col2\" >18.9</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow3_col3\" class=\"data row3 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row4\" class=\"row_heading level0 row4\" >tren_postnorm</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col0\" class=\"data row4 col0\" >13.8</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col1\" class=\"data row4 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col2\" class=\"data row4 col2\" >14.8</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow4_col3\" class=\"data row4 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row5\" class=\"row_heading level0 row5\" >tren_postnorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col0\" class=\"data row5 col0\" >15</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col1\" class=\"data row5 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col2\" class=\"data row5 col2\" >16</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow5_col3\" class=\"data row5 col3\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row6\" class=\"row_heading level0 row6\" >tren_prenorm</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col0\" class=\"data row6 col0\" >13.6</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col1\" class=\"data row6 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col2\" class=\"data row6 col2\" >14.7</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow6_col3\" class=\"data row6 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row7\" class=\"row_heading level0 row7\" >tren_prenorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col0\" class=\"data row7 col0\" >15.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col1\" class=\"data row7 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col2\" class=\"data row7 col2\" >16</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow7_col3\" class=\"data row7 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row8\" class=\"row_heading level0 row8\" >tren_prior_30M_kl</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col0\" class=\"data row8 col0\" >18.6</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col1\" class=\"data row8 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col2\" class=\"data row8 col2\" >19.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow8_col3\" class=\"data row8 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row9\" class=\"row_heading level0 row9\" >tren_prior_30M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col0\" class=\"data row9 col0\" >18.6</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col1\" class=\"data row9 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col2\" class=\"data row9 col2\" >19.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow9_col3\" class=\"data row9 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row10\" class=\"row_heading level0 row10\" >tren_prior_3M_kl</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col0\" class=\"data row10 col0\" >18.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col1\" class=\"data row10 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col2\" class=\"data row10 col2\" >19.3</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow10_col3\" class=\"data row10 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bblevel0_row11\" class=\"row_heading level0 row11\" >tren_prior_3M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col0\" class=\"data row11 col0\" >18.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col1\" class=\"data row11 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col2\" class=\"data row11 col2\" >19.5</td>\n",
       "                        <td id=\"T_1cd8ce62_8a01_11ea_94fd_d0509993d9bbrow11_col3\" class=\"data row11 col3\" >0.2</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_pair(\"tren\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 26.7%, transparent 26.7%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.6%, transparent 25.6%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.8%, transparent 37.8%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 62.2%, transparent 62.2%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 61.5%, transparent 61.5%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 73.3%, transparent 73.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 75.0%, transparent 75.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 71.8%, transparent 71.8%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 20.5%, transparent 20.5%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 4.4%, transparent 4.4%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 37.8%, transparent 37.8%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 50.0%, transparent 50.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.6%, transparent 25.6%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 91.1%, transparent 91.1%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 92.3%, transparent 92.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 33.3%, transparent 33.3%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col0 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col1 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 25.0%, transparent 25.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col2 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 100.0%, transparent 100.0%);\n",
       "        }    #T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col3 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg,#d65f5f 66.7%, transparent 66.7%);\n",
       "        }</style><table id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" colspan=2>dev</th>        <th class=\"col_heading level0 col2\" colspan=2>test</th>    </tr>    <tr>        <th class=\"blank level1\" ></th>        <th class=\"col_heading level1 col0\" >mean</th>        <th class=\"col_heading level1 col1\" ><lambda></th>        <th class=\"col_heading level1 col2\" >mean</th>        <th class=\"col_heading level1 col3\" ><lambda></th>    </tr>    <tr>        <th class=\"index_name level0\" >model</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row0\" class=\"row_heading level0 row0\" >entr_base</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col0\" class=\"data row0 col0\" >12.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col1\" class=\"data row0 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col2\" class=\"data row0 col2\" >11.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow0_col3\" class=\"data row0 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row1\" class=\"row_heading level0 row1\" >entr_base_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col0\" class=\"data row1 col0\" >12.7</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col1\" class=\"data row1 col1\" >0</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col2\" class=\"data row1 col2\" >11.5</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow1_col3\" class=\"data row1 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row2\" class=\"row_heading level0 row2\" >entr_base_ls</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col0\" class=\"data row2 col0\" >13.8</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col1\" class=\"data row2 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col2\" class=\"data row2 col2\" >12.6</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow2_col3\" class=\"data row2 col3\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row3\" class=\"row_heading level0 row3\" >entr_base_ls_fusion_3M</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col0\" class=\"data row3 col0\" >14.3</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col1\" class=\"data row3 col1\" >0.3</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col2\" class=\"data row3 col2\" >13</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow3_col3\" class=\"data row3 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row4\" class=\"row_heading level0 row4\" >entr_postnorm</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col0\" class=\"data row4 col0\" >11</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col1\" class=\"data row4 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col2\" class=\"data row4 col2\" >10.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow4_col3\" class=\"data row4 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row5\" class=\"row_heading level0 row5\" >entr_postnorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col0\" class=\"data row5 col0\" >12.5</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col1\" class=\"data row5 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col2\" class=\"data row5 col2\" >11</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow5_col3\" class=\"data row5 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row6\" class=\"row_heading level0 row6\" >entr_prenorm</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col0\" class=\"data row6 col0\" >11.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col1\" class=\"data row6 col1\" >0.4</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col2\" class=\"data row6 col2\" >10.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow6_col3\" class=\"data row6 col3\" >0.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row7\" class=\"row_heading level0 row7\" >entr_prenorm_ls</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col0\" class=\"data row7 col0\" >12.7</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col1\" class=\"data row7 col1\" >0.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col2\" class=\"data row7 col2\" >11.2</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow7_col3\" class=\"data row7 col3\" >0.3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row8\" class=\"row_heading level0 row8\" >entr_prior_3M_kl</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col0\" class=\"data row8 col0\" >15.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col1\" class=\"data row8 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col2\" class=\"data row8 col2\" >13.8</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow8_col3\" class=\"data row8 col3\" >0.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bblevel0_row9\" class=\"row_heading level0 row9\" >entr_prior_3M_kl_ls</th>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col0\" class=\"data row9 col0\" >15.5</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col1\" class=\"data row9 col1\" >0.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col2\" class=\"data row9 col2\" >14.1</td>\n",
       "                        <td id=\"T_1cd8ce63_8a01_11ea_94fd_d0509993d9bbrow9_col3\" class=\"data row9 col3\" >0.2</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_pair(\"entr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('classic')\n",
    "from matplotlib import rc\n",
    "\n",
    "font_size = 11\n",
    "nice_fonts = {\n",
    "        # Use LaTeX to write all text\n",
    "        \"text.usetex\": True,\n",
    "        \"font.family\": \"serif\",\n",
    "        # Use 10pt font in plots, to match 10pt font in document\n",
    "        \"axes.labelsize\": font_size,\n",
    "        \"font.weight\": \"bold\",\n",
    "        \"axes.labelweight\": \"bold\",\n",
    "        \"font.size\": font_size,\n",
    "        # Make the legend/label fonts a little smaller\n",
    "        \"legend.fontsize\": font_size,\n",
    "        \"xtick.labelsize\": font_size,\n",
    "        \"ytick.labelsize\": font_size,\n",
    "}\n",
    "rc('text', usetex=True)\n",
    "sns.set_style(\"white\", {\n",
    "    \"font.family\": \"serif\",\n",
    "    \"font.serif\": [\"CMU Serif\", \"Times\", \"serif\"],\n",
    "})\n",
    "plt.rcParams.update(nice_fonts)\n",
    "\n",
    "def get_model_subsamples(data, name, dataset):\n",
    "    points = []\n",
    "    points.append(data[data[\"model\"]==f\"{name}_10000\"].mean()[dataset])\n",
    "    points.append(data[data[\"model\"]==f\"{name}_50000\"].mean()[dataset])\n",
    "    points.append(data[data[\"model\"]==f\"{name}_100000\"].mean()[dataset])\n",
    "    points.append(data[data[\"model\"]==f\"{name}\"].mean()[dataset])\n",
    "    return points\n",
    "\n",
    "def plot_model_efficiency( metric):\n",
    "    figsize = (4., 3.9)\n",
    "    fig, ax = plt.subplots(1, 1, figsize=figsize)\n",
    "\n",
    "    ax.plot(get_model_subsamples(df, \"deen_base_ls\", metric), label=r\"Base + \\textsc{LS}\", linestyle='--', linewidth=1, marker='o', color='b')\n",
    "#     ax.plot(get_model_subsamples(df, \"deen_prior_3M_kl\",metric), label=r\"\\textsc{LM}-prior 3M\", linestyle='--', linewidth=1, marker='s', color='r')\n",
    "    ax.plot(get_model_subsamples(df, \"deen_prior_30M_kl\",metric), label=r\"Base + Prior\", linestyle='--', linewidth=1, marker='P', color='g')\n",
    "#     plt.plot(get_model_subsamples(df, \"deen_prior_30M_kl_ls\",metric), label='PRIOR(30)+LS', linestyle='--', linewidth=1, marker='P', color='r')\n",
    "#     plt.title(\"Data efficiency\")\n",
    "    plt.xticks([0,1,2,3], [\"10K\", \"50K\", \"100K\", \"full\"])\n",
    "#     plt.xticks([0,1,2], [\"50K\", \"100K\", \"full (275K)\"])\n",
    "    plt.xlabel(\"parallel data\")\n",
    "    plt.ylabel(\"BLEU\", labelpad=0.6)\n",
    "#     Axes.set_ylabel(self, ylabel, fontdict=None, labelpad=None, **kwargs)[source]\n",
    "    plt.legend(loc='upper left', handlelength=3)\n",
    "    plt.margins(x=.025)\n",
    "    plt.margins(y=.025)\n",
    "#     plt.xlim(0, 6)\n",
    "#     plt.ylim(0, 0.8)\n",
    "    fig.tight_layout()\n",
    "\n",
    "    plt.savefig(f'efficiency_{metric}.pdf', bbox_inches='tight', format=\"pdf\",pad_inches = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae3fdf020e564bf388cdc5f615bcf073",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_model_efficiency(\"test\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
